package controller;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;

import model.Page;

public class Spider {
	
	@SuppressWarnings("serial")
	public final static Set<String> VALID_PAGE_TYPE = 
		new HashSet<String>(){{ add(".html"); add(".htm"); add(".txt"); }};
		
	public final static String DONT_GO_HERE = "http://questioneverything.typepad.com/";
	
	public static LinkedBlockingQueue<Page> pageBuffer;
	public static LinkedBlockingQueue<URI> urlBuffer;
	public static LinkedBlockingQueue<Page> DATA_PAGE_LIST;
	public static Map<String,AtomicInteger> keywordsMap;
	public static Set<URI> urlHistory;
	
	//public static Map<String, AtomicInteger> wordCount = new HashMap<String, AtomicInteger>();
	
	public static AtomicInteger maxPages;
	public static AtomicInteger processedPages;
	public static AtomicInteger retrievedPages;
		
	public static void main(String... args) {
		init(10000);
		
		addKeyWord("biology", "carrot", "science");
		
		String seed = "http://faculty.washington.edu/gmobus/";
		MultiThread multi = null;
		long before = System.nanoTime();
		
		try {
			//new SingleThread(seed).execute();

			multi = new MultiThread(seed, maxPages.get());
			multi.setParserThreadCount(200);
			multi.setRetrieverThreadCount(120);
			multi.execute();
			multi = null;
			
		} catch (final URISyntaxException e) {
			// the url seed is in a bad format. 
			// need to decide what to do next
			e.printStackTrace();
		}
		
		double seconds = (double)(System.nanoTime() - before) / 1000000000.0;
		
		System.out.println("\nTotal Time in seconds: " + seconds);
		System.out.printf("\nURL Buffer Size: %-5d Page Buffer Size: %-5d Processed Pages: %-5d", 
				Spider.urlBuffer.size(), Spider.pageBuffer.size(), Spider.processedPages.get());
		//multi = null;
		//System.gc();
		System.exit(0);
		
	}
	
	/**
	 * This method instantiates the various collections and
	 * class variables used by a parsing object. This
	 * method also turns off the Jericho HTML Page Parser
	 * logging data.
	 * 
	 * @param pages the max number of pages to process.
	 */
	private static void init(final int pages) {
		// turn off parser debug info
		net.htmlparser.jericho.Config.LoggerProvider = 
			net.htmlparser.jericho.LoggerProvider.DISABLED;
		
		retrievedPages = new AtomicInteger();
		processedPages = new AtomicInteger();
		maxPages = new AtomicInteger(pages);
		
		pageBuffer = new LinkedBlockingQueue<Page>();
		urlBuffer = new LinkedBlockingQueue<URI>();
		DATA_PAGE_LIST = new LinkedBlockingQueue<Page>();
		keywordsMap = new HashMap<String,AtomicInteger>();
		urlHistory = new HashSet<URI>(6000);
		
		try {
			final BufferedWriter out =
				new BufferedWriter(new FileWriter("reporterOutput.txt",false));
			out.write("");
			out.close();
		} catch(final IOException e) {
			// what happens if we get here?
		}
	}
	
	/**
	 * This method takes a word and places into a 
	 * keyword list and initializes a counter for
	 * that keyword.
	 * 
	 * @param word the word value
	 */
	private static void addKeyWord(final String ... params) {
		for(String str : params) {
			keywordsMap.put(str, new AtomicInteger(0));
		}
	}
}


